/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
import java.util.*;
 
/**
 *
 * @author indu.chawla
 */

import com.aliasi.tokenizer.EnglishStopTokenizerFactory;
import com.aliasi.tokenizer.IndoEuropeanTokenizerFactory;
import com.aliasi.tokenizer.LowerCaseTokenizerFactory;
import com.aliasi.tokenizer.NGramTokenizerFactory;
import com.aliasi.tokenizer.PorterStemmerTokenizerFactory;
import com.aliasi.tokenizer.Tokenizer;
import com.aliasi.tokenizer.TokenizerFactory;

public class preprocess {
    
    IndoEuropeanTokenizerFactory ietf = IndoEuropeanTokenizerFactory.INSTANCE;
        TokenizerFactory f2=new LowerCaseTokenizerFactory(ietf);
        EnglishStopTokenizerFactory tokFact=new EnglishStopTokenizerFactory(f2); 
   
        PorterStemmerTokenizerFactory stemFact = new PorterStemmerTokenizerFactory(ietf) ; 
        PorterStemmerTokenizerFactory finalFact = new PorterStemmerTokenizerFactory(tokFact) ; 
        
  
        
        
        //This method will remove general stop words from
        public String remove_SW_and_STEM(String input)
        {
                        String output="";
                        char temp[]=input.toCharArray();
                        Tokenizer tokenizer = finalFact.tokenizer(temp,0,temp.length);
                        
                        
                        int i=0;
                        for(String itr:tokenizer)
                        {
                                //System.out.println("itr="+i+"="+itr);
                                output+=itr+" ";
                                i++;
                        }

                        
                        return output;
                                
        }//remove       
        
 //This method will remove general stop words from
public String remove_stop_words(String input)
{
                String output="";
                char temp[]=input.toCharArray();
                Tokenizer tokenizer = tokFact.tokenizer(temp,0,temp.length);
                
                
                //int i=0;
                for(String itr:tokenizer)
                {
                        //System.out.println(i+"="+itr);
                        output+=itr+" ";
                        //i++;
                }

                
                return output;
                        
}//remove

// Applying stemming on words, used to stem camel cased words for baseline approach
public String apply_stemming(String input) 
{
  String output = "";
  char temp[] = input.toCharArray();
  Tokenizer tokenizer = stemFact.tokenizer(temp, 0, temp.length);
  for(String itr:tokenizer )
  {
        output = output + itr+" ";  
  }
  
  return output;
}


 public String s_w1[]={        "public","static","void","java","svn","user","exact","i","$","may", 
                                              "please","forum","refer","like","done","code","need","needs","away","using",
                                              "etc","then","should","now","even","though","long","ago","something",
                                              "various","same","both","branch","branches","trunk","just","another","see","via",
                                              "whether","bugs","bug","too","due","cannot","why","&amp",
                                              "a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s",
                                              "t","u","v","w","x","y","z",
                                              "A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S",
                                              "T","U","V","W","X","Y","Z",
                                              "0","1","2","3","4","5","6","7","8","9"};



 public String remove_SW(String input) 
{
        //System.out.println("Input="+input); 
        
        String output="";
         
         String t[]=input.split("\\s+");
         java.util.List<String> list=Arrays.asList(t);
         Vector<String> temp=new Vector<String>(list);
         
         for(int i=0;i<s_w1.length;i++)
          {
                 for(int j=0;j<temp.size();j++)
                 {
                         if(s_w1[i].equalsIgnoreCase(temp.get(j).toLowerCase()))
                         {
                                temp.remove(j); 
                                j--;
                                
                         }//if
                 }//for
          }//for
         
         for(int i=0;i<temp.size();i++)
         {
                 output+=temp.get(i)+" ";
         }
         
         //System.out.println("Output="+output);
         
         return output;
         
}//function

 
 public String clean(String content)
        {
                        String cleanContent=content;
                        //1. Remove stop words
                        //2. Remove special chars . : - ( ) [ ] ? * = ^ % $ #
                        //3. Remove all the digits 
                        
                        cleanContent=cleanContent.replaceAll("~", " ");
                        cleanContent=cleanContent.replaceAll("`", " ");
                        cleanContent=cleanContent.replaceAll("!", " ");
                        cleanContent=cleanContent.replaceAll("@", " ");
                        cleanContent=cleanContent.replaceAll("#", " ");
                        //cleanContent=cleanContent.replaceAll("\\$", " ");
                        cleanContent=cleanContent.replaceAll("%", " ");
                        cleanContent=cleanContent.replaceAll("\\^", " ");
                        cleanContent=cleanContent.replaceAll("&amp"," ");
                        cleanContent=cleanContent.replaceAll("&", " ");
                        cleanContent=cleanContent.replaceAll("\\*", " ");
                        cleanContent=cleanContent.replaceAll("\\(", " ");
                        cleanContent=cleanContent.replaceAll("\\)", " ");
                        cleanContent=cleanContent.replaceAll("_", " ");
                        cleanContent=cleanContent.replaceAll("-", " ");
                        cleanContent=cleanContent.replaceAll("\\+", " ");
                        cleanContent=cleanContent.replaceAll("=", " ");
                        cleanContent=cleanContent.replaceAll("<", " ");
                        cleanContent=cleanContent.replaceAll(">", " ");
                        cleanContent=cleanContent.replaceAll("\\?", " ");
                        cleanContent=cleanContent.replaceAll("\\.", " ");
                        cleanContent=cleanContent.replaceAll(",", " ");
                        cleanContent=cleanContent.replaceAll("/", " ");
                        cleanContent=cleanContent.replaceAll(";", " ");
                        cleanContent=cleanContent.replaceAll(":", " ");
                        cleanContent=cleanContent.replaceAll("'", " ");
                        cleanContent=cleanContent.replaceAll("\"", " ");
                        cleanContent=cleanContent.replaceAll("\\{", " ");
                        cleanContent=cleanContent.replaceAll("\\[", " ");
                        cleanContent=cleanContent.replaceAll("}", " ");
                        cleanContent=cleanContent.replaceAll("]", " ");
                        cleanContent=cleanContent.replaceAll("\\|", " ");
                        cleanContent=cleanContent.replaceAll("\\\\", " ");
                        cleanContent=cleanContent.replaceAll("'", " ");
                        cleanContent = cleanContent.replaceAll("\"", " ");
                       // cleanContent = cleanContent.replaceAll(".", " ");//dangerous will make ur string null :(
                
                        cleanContent = cleanContent.trim();
                        
                        //replace digit, Not a comment BTW 
                        /*
                        String pattern = "(\\d+)";
                    Pattern r = Pattern.compile(pattern);
                    Matcher m = r.matcher(cleanContent);
                    ArrayList<String> list=new ArrayList<String>();
                    while(m.find()) {
                        String s = m.group(0);
                        //System.out.println(s);
                        cleanContent=cleanContent.replaceAll(s," ");
                        m=r.matcher(cleanContent);
                        //list.add(s);
                    } 
             */
                
                    /*pattern = "(\\s+[a-zA-Z]\\s+ )";
                    r = Pattern.compile(pattern);
                    m = r.matcher(cleanContent);
                    while(m.find()) {
                        String s = m.group(0);
                        System.out.println(s);
                        cleanContent=cleanContent.replaceAll(s," ");
                        m=r.matcher(cleanContent);
                        System.out.println(s);
                        //ProbCalculation.intr();
                        //list.add(s);
                    } */
                    
                        ///@@Testing to remove digits and single alphabets
                        cleanContent = cleanContent.replaceAll("[0-9]"," ");
                		cleanContent = cleanContent.replaceAll(" [a-z] ", " ");
                        
                    cleanContent = cleanContent.replaceAll("\\s+", " ");
                    cleanContent =  cleanContent.replaceAll("\\s+[a-zA-Z]\\s+", " ");
                    cleanContent = cleanContent.replaceAll("\\s+", " ");

                        //for (int i=0;i<list.size();i++)
                    //cleanContent=cleanContent.replace(list.get(i), " ");
                
                    
                        return cleanContent;                    
}

public String camelCasing(String title) {
	// TODO Auto-generated method stub
	return null;
}

}
